package org.dg.iag.dao;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
import java.util.logging.Logger;

import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.dg.iag.crawl.interf.StoreInterface;
import org.dg.iag.entity.FeedTags;
import org.dg.iag.entity.ImportLogStatus;
import org.dg.iag.entity.Log;
import org.dg.iag.entity.User;
import org.dg.iag.util.ApplicationConstants;
import org.dg.iag.util.ImportThread;
import org.dg.iag.util.StorageUtil;
import org.restlet.Response;
import org.restlet.data.CharacterSet;
import org.restlet.data.ClientInfo;
import org.restlet.data.Encoding;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Method;
import org.restlet.data.Preference;
import org.restlet.data.Protocol;
import org.restlet.engine.application.Decoder;
import org.restlet.ext.xstream.XstreamRepresentation;
import org.restlet.representation.Representation;
import org.restlet.resource.ClientResource;
import org.restlet.resource.ResourceException;

import com.thoughtworks.xstream.XStream;
import com.thoughtworks.xstream.io.json.JettisonMappedXmlDriver;

public class FeedDAO {
	FeedTags feeds			= null;
	XStream xstream 	= null;
	
	Logger logger	= Logger.getLogger(FeedDAO.class.getName());
	
	private boolean saveInTempFolderFirst;
	
	public FeedDAO() {
		xstream		= new XstreamRepresentation<User>(new User("test", "test")).getXstream();
		this.saveInTempFolderFirst	= true;
	}
	
	public FeedDAO(boolean saveInTempFolderFirst) {
		this();
		this.saveInTempFolderFirst	= saveInTempFolderFirst;
	}
	
	public FeedTags retrieveFeedTags() {

		ClientResource cr = getClientResource(ApplicationConstants.IATI_API_REST_FULL_PATH, Protocol.HTTP, Method.GET);

		String text = null;
		try {
				text =	cr.get().getText();

		} catch (Exception e) {
			e.printStackTrace();
		}
		if(text == null) return null;
		
		
		String jsonText = "{\"feedtags\":{\"string\":"+text+"}}";
		XStream xstream = new XStream(new JettisonMappedXmlDriver());
		xstream.alias("feedtags", FeedTags.class);
		xstream.aliasField("feed", FeedTags.class, "feedTags");
		xstream.addImplicitCollection(FeedTags.class, "feedTags");

		feeds = (FeedTags)xstream.fromXML(jsonText);

		cr.release();
		return feeds;
	}

	private ClientResource getClientResource(String iatiApiRestFullPath, Protocol protocol, Method method) {
		ClientResource cr = new ClientResource(iatiApiRestFullPath);
		cr.setProtocol(protocol);
		cr.setMethod(method);
		ClientInfo clientInfo = cr.getClientInfo();
	    List<Preference<Encoding>> preflist = clientInfo.getAcceptedEncodings();
	    preflist.add(new Preference<Encoding>(Encoding.GZIP));
	    
	    
	    //TODO remove the lines below
	    //Client client = new Client(InitialServer.configureContext(true), Protocol.HTTP);
	   // client.setConnectTimeout(5000);
	    //cr.setNext(client);
	    
		return cr;
	}
	
	
	public String getFeedAsText(String url, Log log){
		String result = null;
		Representation r = null;
		
		for (int i=0; i< ApplicationConstants.NUM_OF_RETRIES_ON_FEED_FAIL ; i++) {
			logger.info( "start time get REPRESENTATION ... "+url+" ..."+ " TRY: " + i + "..." + System.currentTimeMillis());
			ClientResource cr = getClientResource(url,Protocol.HTTP, Method.GET);
			try {
				r	= cr.get(MediaType.TEXT_XML);
				log.setZipped(checkGzipClientResource(cr));
				
				if(r!=null){
					for (int j=0; j< ApplicationConstants.NUM_OF_RETRIES_ON_FEED_FAIL ; j++) {
						try {
							logger.info("start time get FEED ... "+url+" ..."+ "TRY:" + j + "..." + System.currentTimeMillis());
							result = r.getText();
							int start	= result.indexOf("<");
							if (start > 0)
								result	= result.substring(start);

							logger.info("end time get FEED ... "+url+" ..."+ System.currentTimeMillis());
							break;
						} catch (IOException e) {
							e.printStackTrace();
							continue;
						}
						catch (ResourceException e) {
							e.printStackTrace();
							continue;
						}
						catch (Exception e) {
							e.printStackTrace();
							logger.info("Recovering after Exception. NOT trying again... "+url);
							result = null;
							break;
						}
					}
					r.exhaust();
					r.release();
				}
				break;
			} 
			catch (ResourceException e) {
				e.printStackTrace();
				try {
					Thread.sleep(20000);
					continue;
				} catch (InterruptedException e1) {
					e1.printStackTrace();
					break;
				}
			}
			catch (Exception e) {
				e.printStackTrace();
				break;
			}
			finally{
				System.out.println("RELEASING CLIENT RESOURCE !!");
				cr.release();
			}
		}
		

		logger.info("end time get representation ... "+url+" ..."+ System.currentTimeMillis());
		
		return result;
}

	
	public Representation getFeedAsRepresentation(String url){
		Representation r = null;
		
		ClientResource cr = getClientResource(url,Protocol.HTTP, Method.GET);
		
		System.out.println("start time get representation ... "+url+" ..."+ System.currentTimeMillis());
		try {
			r	= cr.get();
		} catch (ResourceException e) {
			e.printStackTrace();
			System.out.println("Trying again..."+url);
			try{
				cr = getClientResource(url,Protocol.HTTP, Method.GET);
				r	= cr.get();
			}catch(Exception ee){
				ee.printStackTrace();
				System.out.println("Error after another try..."+url);
			}
		}
		catch (Exception e) {
			e.printStackTrace();
		}
		System.out.println("end time get representation ... "+url+" ..."+ System.currentTimeMillis());
		
		return r;
}

	
	
	public String getFeedURL(String tag){
		ClientResource cr = getClientResource(ApplicationConstants.IATI_API_REST_FULL_PATH+tag,
				Protocol.HTTP, Method.GET);
		String text = null;
		try {
			//text = cr.getResponse().getEntity().getText();
			text = cr.get().getText();
		} catch (IOException e) {
			e.printStackTrace();
		}
		if(text == null) return null;

		JSONObject json = null;
		try {
			json = new JSONObject(text);
		} catch (JSONException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		String feedUrl = null;
		try {
			feedUrl = json.getString("download_url");
		} catch (JSONException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		System.out.println("feedUrl: " + feedUrl);
		return feedUrl;
	}

	public void storeAllFeedsTest(ImportThread thisThread, ArrayList<String> tags){
		
		long currentTimeMillis = System.currentTimeMillis();
		int currentTagNo=0;
		StoreInterface feedStore = StorageUtil.getFeedStore("exist");
		Integer totalNoTags = tags.size();
		ImportLogStatus importLogStatus = ImportLogStatus.getInstance();
		importLogStatus.setTotalNoTags(totalNoTags);
		TreeSet<String> tagErrorQueue = new TreeSet<String>();
		for (Iterator<String> it = tags.iterator(); it.hasNext();) {
			try{
				String tag = (String) it.next();
				currentTagNo++;
				//if ( currentTagNo == 9 ) break; // Just for testing how the import ends
				System.out.println("Processing the tag "+tag+" -- "+currentTagNo+" out of "+totalNoTags);
				this.storeFeed(currentTimeMillis, currentTagNo, importLogStatus, tag, tagErrorQueue);
				if ( thisThread != null && thisThread.stop.get() ) {
					importLogStatus.setFinished(true);
					feedStore.finish();
					return;
				}
			}catch(Exception ex){
				ex.printStackTrace();
			}
		}
		
		importLogStatus.setFinished(true);
		feedStore.finish();
		
	}
	
	public void storeAllFeeds(ImportThread thisThread) {
		this.retrieveFeedTags();
		long currentTimeMillis = System.currentTimeMillis();
		int currentTagNo=0;
		StoreInterface feedStore = StorageUtil.getFeedStore("exist");
		Integer totalNoTags = this.feeds.getFeedTags().size();
		ImportLogStatus importLogStatus = ImportLogStatus.getInstance();
		importLogStatus.setTotalNoTags(totalNoTags);
		TreeSet<String> tagErrorQueue = new TreeSet<String>();
		for (Iterator<String> it = this.feeds.getFeedTags().iterator(); it.hasNext();) {
			try{
				String tag = (String) it.next();
				currentTagNo++;
				//if(currentTagNo == 649) 
					//System.out.println("found something");
				//if(currentTagNo == 10 ||currentTagNo == 11) continue; 
				//if ( currentTagNo == 9 ) break; // Just for testing how the import ends
				System.out.println("Processing the tag "+tag+" -- "+currentTagNo+" out of "+totalNoTags);
				this.storeFeed(currentTimeMillis, currentTagNo, importLogStatus, tag, tagErrorQueue);
				if ( thisThread != null && thisThread.stop.get() ) {
					importLogStatus.setFinished(true);
					feedStore.finish();
					return;
				}
			}catch(Exception ex){
				ex.printStackTrace();
			}
		}
		
		//retrying the tags that have error second time
//		ArrayList<String> tagErrorListAux = new ArrayList<String>();
//		tagErrorListAux.addAll(tagErrorQueue);
//		tagErrorQueue.clear();
//		System.out.println("...................................................");
//		currentTagNo=0;
//		System.out.println("Retrying to import the list of tags that crashed...");
//		for (Iterator<String> it = tagErrorListAux.iterator(); it.hasNext();) {
//			String tag = (String) it.next();
//			currentTagNo++;
//			System.out.println("Processing the tag "+tag+" -- "+currentTagNo+" out of "+tagErrorListAux.size());
//				this.storeFeed(currentTimeMillis, currentTagNo, importLogStatus, tag, tagErrorQueue);
//		}
		
		importLogStatus.setFinished(true);
		feedStore.finish();
		 
	}
	
	public void testGzip(){
		StoreInterface feedStore = StorageUtil.getFeedStore("exist");
		String url =  "http://ec.europa.eu/europeaid/files/iati/EU_QNE.xml";
		ClientResource cr = getClientResource(url,Protocol.HTTP, Method.GET);
		Representation representation = cr.get();
		if(checkGzipClientResource(cr))
			System.out.println("url "+url + " has compression GZIP");
		else System.out.println("url "+url + " has NO compression GZIP");
		
		System.out.println("done: ");
	}

	private boolean checkGzipClientResource(ClientResource cr) {
		Map<String, Object> responseAttributes = cr.getResponseAttributes();
		Form f = (Form)responseAttributes.get("org.restlet.http.headers");
		Map<String, String> valuesMap = f.getValuesMap();
		if(valuesMap.containsKey("Content-Encoding")){
			if("gzip".compareTo(valuesMap.get("Content-Encoding").toString().toLowerCase()) ==0)
				return true;
		}
		return false;
	}

	private void storeFeed(long currentTimeMillis, int currentTagNo,ImportLogStatus importLogStatus, String tag, TreeSet<String> tagErrorQueue) {
		String url = getFeedURL(tag);
		StoreInterface feedStore = StorageUtil.getFeedStore("exist");
		feedStore.setFirstInTempLocation(this.saveInTempFolderFirst);
		
		String text		= null;
		Log logAux = new Log();
		try {
			text = this.getFeedAsText(url, logAux);
		}
		catch (Exception e) {
			e.printStackTrace();
		}
		try {
			Log log = feedStore.store(currentTimeMillis, tag, url, null, text, logAux.getZipped());
			//populating the import log stauts
			if(log.getImportStatus() == 0) tagErrorQueue.add(tag);
			populateImportLogStatus(importLogStatus, log, currentTagNo, log.getTimeMillis());
		}
		catch (Exception e) {
			e.printStackTrace();
		}
	}

	public boolean storeFeeds(List<String> tagsList, long currentTimeMillis, TreeSet<String> tagErrorQueue){
		this.retrieveFeedTags();
		int currentTagNo=0;
		Integer totalNoTags = tagsList.size();
		ImportLogStatus importLogStatus = ImportLogStatus.getInstance();
		importLogStatus.setTotalNoTags(totalNoTags);
		TreeSet<String> iatiTags = new TreeSet<String>(this.feeds.getFeedTags());
		if(iatiTags == null || iatiTags.isEmpty()) return false;
		for (Iterator it = tagsList.iterator(); it.hasNext();) {
			String tag = (String) it.next();
			currentTagNo++;
			System.out.println("Processing the tag "+tag+" -- "+currentTagNo+" out of "+totalNoTags);
			if(iatiTags.contains(tag))
				this.storeFeed(currentTimeMillis, currentTagNo, importLogStatus, tag, tagErrorQueue);
		}
		
		//retrying the tags that have error second time
		System.out.println("Retrying the tags that have errors ");
		ArrayList<String> tagErrorListAux = new ArrayList<String>();
		tagErrorListAux.addAll(tagErrorQueue);
		tagErrorQueue.clear();
		for (Iterator it = tagErrorListAux.iterator(); it.hasNext();) {
			String tag = (String) it.next();
			currentTagNo++;
			System.out.println("Processing the tag "+tag+" -- "+currentTagNo+" out of "+totalNoTags);
			if(iatiTags.contains(tag))
				this.storeFeed(currentTimeMillis, currentTagNo, importLogStatus, tag, tagErrorQueue);
		}
		importLogStatus.setFinished(true);
		return true;
	}
	
	
	private synchronized void populateImportLogStatus(ImportLogStatus importLogStatus, Log log, int currentTagNo, Long timeMillis) {
		// TODO Auto-generated method stub
		importLogStatus.setCurrentTagNo(currentTagNo);
		importLogStatus.setLog(log);
		importLogStatus.setGroupId(timeMillis);
	}
	
	//create xml from json
//	HierarchicalStreamDriver driver = new JettisonMappedXmlDriver();
//	StringReader reader = new StringReader(jsonText);
//	HierarchicalStreamReader hsr = driver.createReader(reader);
//	StringWriter writer = new StringWriter();
//	new HierarchicalStreamCopier().copy(hsr, new PrettyPrintWriter(writer));
//	try {
//		writer.close();
//	} catch (IOException e) {
//		e.printStackTrace();
//	}
//	String xmlText = writer.toString();
	
	
//check url if gzip	
//	URL url;
//	try {
//		url = new URL("http://ec.europa.eu/europeaid/files/iati/EU_QNE.xml");
//		HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
//		urlConnection.setRequestProperty("Accept-Encoding", "gzip, deflate");
//
//
//	//InputStream input = urlConnection.getInputStream();
//	String encoding = urlConnection.getContentEncoding();
//	Map<String, List<String>> headerFields = urlConnection.getHeaderFields();
//	String responseMessage = urlConnection.getResponseMessage();
//	urlConnection.getContentType();
//	System.out.println("gata:: "+headerFields);
//	} catch (Exception e) {
//		// TODO Auto-generated catch block
//		e.printStackTrace();
//	}

	
}
